{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install --upgrade wandb\n",
    "!wandb login 3da7a23df9fd940d985adf808de2b09ceb85f15b\n",
    "\n",
    "import wandb\n",
    "wandb.init(project=\"global-wheat-detection\", name='FasterRCNN with ResNet101 backbone: fold3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install cython\n",
    "!pip install 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'\n",
    "!cp /kaggle/input/rcnnutilswithwandb/engine.py .\n",
    "!cp /kaggle/input/rcnnutilswithwandb/utils.py .\n",
    "!cp /kaggle/input/rcnnutilswithwandb/coco_eval.py .\n",
    "!cp /kaggle/input/rcnnutilswithwandb/coco_utils.py .\n",
    "!cp /kaggle/input/rcnnutilswithwandb/transforms.py ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import ast\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import cv2\n",
    "\n",
    "import torch\n",
    "from PIL import Image\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "import albumentations\n",
    "from albumentations.pytorch.transforms import ToTensorV2\n",
    "\n",
    "from torch import nn\n",
    "import torchvision\n",
    "import torch.utils.data as data_utils\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torchvision.models.detection.faster_rcnn import FastRCNNPredictor\n",
    "from torchvision.models.detection import FasterRCNN\n",
    "from torchvision.models.detection.rpn import AnchorGenerator\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import matplotlib.patches as patches\n",
    "\n",
    "import utils\n",
    "from engine import train_one_epoch, evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Constants\n",
    "TEST_DIR = '/kaggle/input/global-wheat-detection/test'\n",
    "BASE_DIR = '/kaggle/input/gwdaugmented/train'\n",
    "BATCH_SIZE = 2\n",
    "DEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "# our dataset has two classes only - background and wheat heads\n",
    "N_CLASSES = 2\n",
    "N_EPOCHS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>x_min</th>\n",
       "      <th>y_min</th>\n",
       "      <th>x_max</th>\n",
       "      <th>y_max</th>\n",
       "      <th>width</th>\n",
       "      <th>height</th>\n",
       "      <th>area</th>\n",
       "      <th>source</th>\n",
       "      <th>kfold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>834.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>890.0</td>\n",
       "      <td>258.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>226.0</td>\n",
       "      <td>548.0</td>\n",
       "      <td>356.0</td>\n",
       "      <td>606.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>7540.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>377.0</td>\n",
       "      <td>504.0</td>\n",
       "      <td>451.0</td>\n",
       "      <td>664.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>11840.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>834.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>943.0</td>\n",
       "      <td>202.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>11663.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>26.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>261.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>14508.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295533</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>876.0</td>\n",
       "      <td>619.0</td>\n",
       "      <td>960.0</td>\n",
       "      <td>714.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>7980.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295534</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>625.0</td>\n",
       "      <td>549.0</td>\n",
       "      <td>732.0</td>\n",
       "      <td>631.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>8774.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295535</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>749.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>890.0</td>\n",
       "      <td>299.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>10011.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295536</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>410.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>14536.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295537</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>55.0</td>\n",
       "      <td>740.0</td>\n",
       "      <td>149.0</td>\n",
       "      <td>801.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>5734.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>295538 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             image_id  x_min  y_min  x_max  y_max  width  height     area  \\\n",
       "0           b6ab77fd7  834.0  222.0  890.0  258.0   56.0    36.0   2016.0   \n",
       "1           b6ab77fd7  226.0  548.0  356.0  606.0  130.0    58.0   7540.0   \n",
       "2           b6ab77fd7  377.0  504.0  451.0  664.0   74.0   160.0  11840.0   \n",
       "3           b6ab77fd7  834.0   95.0  943.0  202.0  109.0   107.0  11663.0   \n",
       "4           b6ab77fd7   26.0  144.0  150.0  261.0  124.0   117.0  14508.0   \n",
       "...               ...    ...    ...    ...    ...    ...     ...      ...   \n",
       "295533  5e0747034_aug  876.0  619.0  960.0  714.0   84.0    95.0   7980.0   \n",
       "295534  5e0747034_aug  625.0  549.0  732.0  631.0  107.0    82.0   8774.0   \n",
       "295535  5e0747034_aug  749.0  228.0  890.0  299.0  141.0    71.0  10011.0   \n",
       "295536  5e0747034_aug  410.0   13.0  594.0   92.0  184.0    79.0  14536.0   \n",
       "295537  5e0747034_aug   55.0  740.0  149.0  801.0   94.0    61.0   5734.0   \n",
       "\n",
       "           source  kfold  \n",
       "0         usask_1      2  \n",
       "1         usask_1      2  \n",
       "2         usask_1      2  \n",
       "3         usask_1      2  \n",
       "4         usask_1      2  \n",
       "...           ...    ...  \n",
       "295533  arvalis_2      1  \n",
       "295534  arvalis_2      1  \n",
       "295535  arvalis_2      1  \n",
       "295536  arvalis_2      1  \n",
       "295537  arvalis_2      1  \n",
       "\n",
       "[295538 rows x 10 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv(os.path.join('/kaggle/input/gwdaugmented/', 'train.csv'))\n",
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>x_min</th>\n",
       "      <th>y_min</th>\n",
       "      <th>x_max</th>\n",
       "      <th>y_max</th>\n",
       "      <th>width</th>\n",
       "      <th>height</th>\n",
       "      <th>area</th>\n",
       "      <th>source</th>\n",
       "      <th>kfold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>834.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>890.0</td>\n",
       "      <td>258.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>226.0</td>\n",
       "      <td>548.0</td>\n",
       "      <td>356.0</td>\n",
       "      <td>606.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>7540.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>377.0</td>\n",
       "      <td>504.0</td>\n",
       "      <td>451.0</td>\n",
       "      <td>664.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>11840.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>834.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>943.0</td>\n",
       "      <td>202.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>11663.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>26.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>261.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>14508.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295533</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>876.0</td>\n",
       "      <td>619.0</td>\n",
       "      <td>960.0</td>\n",
       "      <td>714.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>7980.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295534</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>625.0</td>\n",
       "      <td>549.0</td>\n",
       "      <td>732.0</td>\n",
       "      <td>631.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>8774.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295535</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>749.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>890.0</td>\n",
       "      <td>299.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>10011.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295536</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>410.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>14536.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295537</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>55.0</td>\n",
       "      <td>740.0</td>\n",
       "      <td>149.0</td>\n",
       "      <td>801.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>5734.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>295538 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             image_id  x_min  y_min  x_max  y_max  width  height     area  \\\n",
       "0           b6ab77fd7  834.0  222.0  890.0  258.0   56.0    36.0   2016.0   \n",
       "1           b6ab77fd7  226.0  548.0  356.0  606.0  130.0    58.0   7540.0   \n",
       "2           b6ab77fd7  377.0  504.0  451.0  664.0   74.0   160.0  11840.0   \n",
       "3           b6ab77fd7  834.0   95.0  943.0  202.0  109.0   107.0  11663.0   \n",
       "4           b6ab77fd7   26.0  144.0  150.0  261.0  124.0   117.0  14508.0   \n",
       "...               ...    ...    ...    ...    ...    ...     ...      ...   \n",
       "295533  5e0747034_aug  876.0  619.0  960.0  714.0   84.0    95.0   7980.0   \n",
       "295534  5e0747034_aug  625.0  549.0  732.0  631.0  107.0    82.0   8774.0   \n",
       "295535  5e0747034_aug  749.0  228.0  890.0  299.0  141.0    71.0  10011.0   \n",
       "295536  5e0747034_aug  410.0   13.0  594.0   92.0  184.0    79.0  14536.0   \n",
       "295537  5e0747034_aug   55.0  740.0  149.0  801.0   94.0    61.0   5734.0   \n",
       "\n",
       "           source  kfold  \n",
       "0         usask_1      2  \n",
       "1         usask_1      2  \n",
       "2         usask_1      2  \n",
       "3         usask_1      2  \n",
       "4         usask_1      2  \n",
       "...           ...    ...  \n",
       "295533  arvalis_2      1  \n",
       "295534  arvalis_2      1  \n",
       "295535  arvalis_2      1  \n",
       "295536  arvalis_2      1  \n",
       "295537  arvalis_2      1  \n",
       "\n",
       "[295538 rows x 10 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WheatDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, df, folds, transforms=None):\n",
    "        self.df = df[df.kfold.isin(folds)].reset_index(drop=True)\n",
    "        self.image_ids = self.df['image_id'].unique()\n",
    "        self.transforms = transforms\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.image_ids)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        image_id = self.image_ids[index]\n",
    "        image = cv2.imread(os.path.join(BASE_DIR, 'train', f'{image_id}.jpg'), cv2.IMREAD_COLOR)\n",
    "        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n",
    "        image /= 255.0\n",
    "\n",
    "        # Convert from NHWC to NCHW as pytorch expects images in NCHW format\n",
    "        image = np.transpose(image, (2, 0, 1))\n",
    "        image = torch.from_numpy(image)\n",
    "        \n",
    "        # Get bbox coordinates for each wheat head(s)\n",
    "        bboxes_df = self.df[self.df['image_id'] == image_id]\n",
    "        boxes, areas = [], []\n",
    "        n_objects = len(bboxes_df)  # Number of wheat heads in the given image\n",
    "\n",
    "        for i in range(n_objects):\n",
    "            x_min = bboxes_df.iloc[i]['x_min']\n",
    "            x_max = bboxes_df.iloc[i]['x_max']\n",
    "            y_min = bboxes_df.iloc[i]['y_min']\n",
    "            y_max = bboxes_df.iloc[i]['y_max']\n",
    "\n",
    "            boxes.append([x_min, y_min, x_max, y_max])\n",
    "            areas.append(bboxes_df.iloc[i]['area'])\n",
    "\n",
    "        boxes = torch.as_tensor(boxes, dtype=torch.int64)\n",
    "        \n",
    "        # Get the labels. We have only one class (wheat head)\n",
    "        labels = torch.ones((n_objects, ), dtype=torch.int64)\n",
    "        \n",
    "        areas = torch.as_tensor(areas)\n",
    "        \n",
    "        # suppose all instances are not crowd\n",
    "        iscrowd = torch.zeros((n_objects, ), dtype=torch.int64)\n",
    "        \n",
    "        target = {\n",
    "            'boxes': boxes,\n",
    "            'labels': labels,\n",
    "            'image_id': torch.tensor([index]),\n",
    "            'area': areas,\n",
    "            'iscrowd': iscrowd\n",
    "        }\n",
    "        \n",
    "        if self.transforms:\n",
    "            result_aug = self.transforms(image=image, bboxes=boxes, labels=labels)\n",
    "            image = result_aug['image'].float()\n",
    "            \n",
    "            target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*result_aug['bboxes'])))).permute(1, 0)\n",
    "\n",
    "        return image, target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(pre_trained=True):\n",
    "    \n",
    "    # Reference: https://stackoverflow.com/questions/58362892/resnet-18-as-backbone-in-faster-r-cnn\n",
    "    resnet_net = torchvision.models.resnet101(pretrained=True) \n",
    "    modules = list(resnet_net.children())[:-2]\n",
    "\n",
    "    backbone = nn.Sequential(*modules)\n",
    "    backbone.out_channels = 2048\n",
    "\n",
    "    # let's make the RPN generate 5 x 3 anchors per spatial\n",
    "    # location, with 5 different sizes and 3 different aspect\n",
    "    # ratios. We have a Tuple[Tuple[int]] because each feature\n",
    "    # map could potentially have different sizes and\n",
    "    # aspect ratios\n",
    "    anchor_generator = AnchorGenerator(sizes=((32, 64, 128, 256, 512),),\n",
    "                                       aspect_ratios=((0.5, 1.0, 2.0),))\n",
    "\n",
    "    # put the pieces together inside a FasterRCNN model\n",
    "    model = FasterRCNN(backbone,\n",
    "                       num_classes=N_CLASSES,\n",
    "                       rpn_anchor_generator=anchor_generator)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "# get the model using our helper function\n",
    "model = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:2854: UserWarning: The default behavior for interpolate/upsample with float scale_factor will change in 1.6.0 to align with other frameworks/libraries, and use scale_factor directly, instead of relying on the computed output size. If you wish to keep the old behavior, please set recompute_scale_factor=True. See the documentation of nn.Upsample for details. \n",
      "  warnings.warn(\"The default behavior for interpolate/upsample with float scale_factor will change \"\n",
      "/opt/conda/conda-bld/pytorch_1587428398394/work/torch/csrc/utils/python_arg_parser.cpp:756: UserWarning: This overload of nonzero is deprecated:\n",
      "\tnonzero(Tensor input, *, Tensor out)\n",
      "Consider using one of the following signatures instead:\n",
      "\tnonzero(Tensor input, *, bool as_tuple)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [0]  [   0/2699]  eta: 2:33:45  lr: 0.000010  loss: 1.7993 (1.7993)  loss_classifier: 0.7014 (0.7014)  loss_box_reg: 0.1093 (0.1093)  loss_objectness: 0.7182 (0.7182)  loss_rpn_box_reg: 0.2704 (0.2704)  time: 3.4183  data: 0.9532  max mem: 5070\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:2854: UserWarning: The default behavior for interpolate/upsample with float scale_factor will change in 1.6.0 to align with other frameworks/libraries, and use scale_factor directly, instead of relying on the computed output size. If you wish to keep the old behavior, please set recompute_scale_factor=True. See the documentation of nn.Upsample for details. \n",
      "  warnings.warn(\"The default behavior for interpolate/upsample with float scale_factor will change \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [0]  [ 100/2699]  eta: 0:23:32  lr: 0.000509  loss: 1.1860 (1.3822)  loss_classifier: 0.4100 (0.4744)  loss_box_reg: 0.2432 (0.1932)  loss_objectness: 0.3311 (0.4982)  loss_rpn_box_reg: 0.1886 (0.2164)  time: 0.5080  data: 0.0101  max mem: 6482\n",
      "Epoch: [0]  [ 200/2699]  eta: 0:22:01  lr: 0.001009  loss: 1.0913 (1.2864)  loss_classifier: 0.3784 (0.4349)  loss_box_reg: 0.3167 (0.2458)  loss_objectness: 0.2610 (0.4048)  loss_rpn_box_reg: 0.1406 (0.2008)  time: 0.5109  data: 0.0110  max mem: 6482\n",
      "Epoch: [0]  [ 300/2699]  eta: 0:20:56  lr: 0.001508  loss: 1.0813 (1.2348)  loss_classifier: 0.3670 (0.4158)  loss_box_reg: 0.3874 (0.2808)  loss_objectness: 0.2047 (0.3514)  loss_rpn_box_reg: 0.1456 (0.1868)  time: 0.5090  data: 0.0103  max mem: 6482\n",
      "Epoch: [0]  [ 400/2699]  eta: 0:19:58  lr: 0.002008  loss: 1.0576 (1.1964)  loss_classifier: 0.3777 (0.4049)  loss_box_reg: 0.3979 (0.3060)  loss_objectness: 0.1526 (0.3096)  loss_rpn_box_reg: 0.1324 (0.1758)  time: 0.5102  data: 0.0101  max mem: 6482\n",
      "Epoch: [0]  [ 500/2699]  eta: 0:19:03  lr: 0.002507  loss: 0.9997 (1.1652)  loss_classifier: 0.3410 (0.3958)  loss_box_reg: 0.3741 (0.3198)  loss_objectness: 0.1427 (0.2800)  loss_rpn_box_reg: 0.1312 (0.1696)  time: 0.5378  data: 0.0142  max mem: 6482\n",
      "Epoch: [0]  [ 600/2699]  eta: 0:18:08  lr: 0.003007  loss: 0.9382 (1.1250)  loss_classifier: 0.3228 (0.3832)  loss_box_reg: 0.3459 (0.3247)  loss_objectness: 0.1273 (0.2554)  loss_rpn_box_reg: 0.1287 (0.1617)  time: 0.5092  data: 0.0102  max mem: 6482\n",
      "Epoch: [0]  [ 700/2699]  eta: 0:17:15  lr: 0.003506  loss: 0.7731 (1.0832)  loss_classifier: 0.2882 (0.3707)  loss_box_reg: 0.3220 (0.3247)  loss_objectness: 0.0854 (0.2339)  loss_rpn_box_reg: 0.0832 (0.1540)  time: 0.5124  data: 0.0103  max mem: 6482\n",
      "Epoch: [0]  [ 800/2699]  eta: 0:16:22  lr: 0.004006  loss: 0.8389 (1.0502)  loss_classifier: 0.3031 (0.3609)  loss_box_reg: 0.3354 (0.3246)  loss_objectness: 0.0868 (0.2172)  loss_rpn_box_reg: 0.1034 (0.1474)  time: 0.5096  data: 0.0101  max mem: 6482\n",
      "Epoch: [0]  [ 900/2699]  eta: 0:15:30  lr: 0.004505  loss: 0.7925 (1.0231)  loss_classifier: 0.2750 (0.3527)  loss_box_reg: 0.2974 (0.3230)  loss_objectness: 0.0931 (0.2043)  loss_rpn_box_reg: 0.1081 (0.1431)  time: 0.5104  data: 0.0105  max mem: 6482\n",
      "Epoch: [0]  [1000/2699]  eta: 0:14:38  lr: 0.005000  loss: 0.7255 (0.9997)  loss_classifier: 0.2687 (0.3459)  loss_box_reg: 0.3029 (0.3215)  loss_objectness: 0.0783 (0.1933)  loss_rpn_box_reg: 0.0812 (0.1390)  time: 0.5108  data: 0.0106  max mem: 6482\n",
      "Epoch: [0]  [1100/2699]  eta: 0:13:46  lr: 0.005000  loss: 0.7304 (0.9805)  loss_classifier: 0.2720 (0.3406)  loss_box_reg: 0.2968 (0.3198)  loss_objectness: 0.0786 (0.1845)  loss_rpn_box_reg: 0.0805 (0.1357)  time: 0.5103  data: 0.0101  max mem: 6482\n",
      "Epoch: [0]  [1200/2699]  eta: 0:12:54  lr: 0.005000  loss: 0.7261 (0.9599)  loss_classifier: 0.2805 (0.3344)  loss_box_reg: 0.2827 (0.3175)  loss_objectness: 0.0716 (0.1760)  loss_rpn_box_reg: 0.0953 (0.1321)  time: 0.5274  data: 0.0131  max mem: 6482\n",
      "Epoch: [0]  [1300/2699]  eta: 0:12:02  lr: 0.005000  loss: 0.7149 (0.9416)  loss_classifier: 0.2561 (0.3291)  loss_box_reg: 0.3006 (0.3150)  loss_objectness: 0.0747 (0.1685)  loss_rpn_box_reg: 0.0830 (0.1290)  time: 0.5113  data: 0.0106  max mem: 6482\n",
      "Epoch: [0]  [1400/2699]  eta: 0:11:10  lr: 0.005000  loss: 0.6646 (0.9239)  loss_classifier: 0.2470 (0.3241)  loss_box_reg: 0.2615 (0.3122)  loss_objectness: 0.0725 (0.1618)  loss_rpn_box_reg: 0.0673 (0.1258)  time: 0.5106  data: 0.0104  max mem: 6482\n",
      "Epoch: [0]  [1500/2699]  eta: 0:10:19  lr: 0.005000  loss: 0.7175 (0.9075)  loss_classifier: 0.2642 (0.3194)  loss_box_reg: 0.2707 (0.3089)  loss_objectness: 0.0721 (0.1559)  loss_rpn_box_reg: 0.0886 (0.1232)  time: 0.5111  data: 0.0106  max mem: 6482\n",
      "Epoch: [0]  [1600/2699]  eta: 0:09:27  lr: 0.005000  loss: 0.6862 (0.8932)  loss_classifier: 0.2543 (0.3156)  loss_box_reg: 0.2751 (0.3059)  loss_objectness: 0.0622 (0.1507)  loss_rpn_box_reg: 0.0848 (0.1210)  time: 0.5096  data: 0.0106  max mem: 6482\n",
      "Epoch: [0]  [1700/2699]  eta: 0:08:35  lr: 0.005000  loss: 0.6531 (0.8802)  loss_classifier: 0.2420 (0.3119)  loss_box_reg: 0.2444 (0.3032)  loss_objectness: 0.0568 (0.1461)  loss_rpn_box_reg: 0.0753 (0.1190)  time: 0.5133  data: 0.0109  max mem: 6482\n",
      "Epoch: [0]  [1800/2699]  eta: 0:07:43  lr: 0.005000  loss: 0.6178 (0.8667)  loss_classifier: 0.2350 (0.3081)  loss_box_reg: 0.2562 (0.3003)  loss_objectness: 0.0623 (0.1415)  loss_rpn_box_reg: 0.0747 (0.1168)  time: 0.5120  data: 0.0110  max mem: 6482\n",
      "Epoch: [0]  [1900/2699]  eta: 0:06:52  lr: 0.005000  loss: 0.6128 (0.8546)  loss_classifier: 0.2345 (0.3048)  loss_box_reg: 0.2376 (0.2975)  loss_objectness: 0.0590 (0.1373)  loss_rpn_box_reg: 0.0738 (0.1149)  time: 0.5214  data: 0.0122  max mem: 6482\n",
      "Epoch: [0]  [2000/2699]  eta: 0:06:00  lr: 0.005000  loss: 0.6657 (0.8431)  loss_classifier: 0.2447 (0.3015)  loss_box_reg: 0.2545 (0.2949)  loss_objectness: 0.0606 (0.1336)  loss_rpn_box_reg: 0.0779 (0.1130)  time: 0.5134  data: 0.0112  max mem: 6482\n",
      "Epoch: [0]  [2100/2699]  eta: 0:05:08  lr: 0.005000  loss: 0.5753 (0.8329)  loss_classifier: 0.2402 (0.2986)  loss_box_reg: 0.2397 (0.2924)  loss_objectness: 0.0503 (0.1303)  loss_rpn_box_reg: 0.0603 (0.1115)  time: 0.5115  data: 0.0107  max mem: 6482\n",
      "Epoch: [0]  [2200/2699]  eta: 0:04:17  lr: 0.005000  loss: 0.5854 (0.8226)  loss_classifier: 0.2262 (0.2958)  loss_box_reg: 0.2317 (0.2899)  loss_objectness: 0.0437 (0.1271)  loss_rpn_box_reg: 0.0683 (0.1099)  time: 0.5093  data: 0.0104  max mem: 6482\n",
      "Epoch: [0]  [2300/2699]  eta: 0:03:25  lr: 0.005000  loss: 0.6461 (0.8139)  loss_classifier: 0.2380 (0.2933)  loss_box_reg: 0.2586 (0.2876)  loss_objectness: 0.0537 (0.1243)  loss_rpn_box_reg: 0.0837 (0.1087)  time: 0.5129  data: 0.0112  max mem: 6482\n",
      "Epoch: [0]  [2400/2699]  eta: 0:02:34  lr: 0.005000  loss: 0.5631 (0.8047)  loss_classifier: 0.2190 (0.2907)  loss_box_reg: 0.2153 (0.2851)  loss_objectness: 0.0474 (0.1216)  loss_rpn_box_reg: 0.0601 (0.1073)  time: 0.5093  data: 0.0106  max mem: 6482\n",
      "Epoch: [0]  [2500/2699]  eta: 0:01:42  lr: 0.005000  loss: 0.5971 (0.7965)  loss_classifier: 0.2154 (0.2883)  loss_box_reg: 0.2337 (0.2831)  loss_objectness: 0.0545 (0.1190)  loss_rpn_box_reg: 0.0873 (0.1061)  time: 0.5128  data: 0.0106  max mem: 6482\n",
      "Epoch: [0]  [2600/2699]  eta: 0:00:51  lr: 0.005000  loss: 0.6026 (0.7883)  loss_classifier: 0.2125 (0.2859)  loss_box_reg: 0.2313 (0.2808)  loss_objectness: 0.0509 (0.1166)  loss_rpn_box_reg: 0.0725 (0.1049)  time: 0.5295  data: 0.0140  max mem: 6482\n",
      "Epoch: [0]  [2698/2699]  eta: 0:00:00  lr: 0.005000  loss: 0.5973 (0.7815)  loss_classifier: 0.2321 (0.2840)  loss_box_reg: 0.2273 (0.2790)  loss_objectness: 0.0497 (0.1146)  loss_rpn_box_reg: 0.0819 (0.1039)  time: 0.4950  data: 0.0104  max mem: 6482\n",
      "Epoch: [0] Total time: 0:23:10 (0.5151 s / it)\n",
      "creating index...\n",
      "index created!\n",
      "Test:  [  0/675]  eta: 0:16:22  model_time: 0.2361 (0.2361)  evaluator_time: 0.4262 (0.4262)  time: 1.4560  data: 0.7782  max mem: 6482\n",
      "Test:  [100/675]  eta: 0:06:31  model_time: 0.1612 (0.1640)  evaluator_time: 0.3867 (0.4886)  time: 0.5812  data: 0.0107  max mem: 6482\n",
      "Test:  [200/675]  eta: 0:05:48  model_time: 0.1622 (0.1636)  evaluator_time: 0.8402 (0.5469)  time: 1.0004  data: 0.0107  max mem: 6482\n",
      "Test:  [300/675]  eta: 0:04:23  model_time: 0.1623 (0.1641)  evaluator_time: 0.4725 (0.5163)  time: 0.6726  data: 0.0105  max mem: 6482\n",
      "Test:  [400/675]  eta: 0:03:07  model_time: 0.1636 (0.1645)  evaluator_time: 0.5295 (0.4971)  time: 0.7512  data: 0.0108  max mem: 6482\n",
      "Test:  [500/675]  eta: 0:01:58  model_time: 0.1632 (0.1645)  evaluator_time: 0.8874 (0.4940)  time: 1.1069  data: 0.0114  max mem: 6482\n",
      "Test:  [600/675]  eta: 0:00:51  model_time: 0.1685 (0.1648)  evaluator_time: 0.2838 (0.5063)  time: 0.5231  data: 0.0117  max mem: 6482\n",
      "Test:  [674/675]  eta: 0:00:00  model_time: 0.1657 (0.1647)  evaluator_time: 0.1732 (0.4901)  time: 0.3617  data: 0.0114  max mem: 6482\n",
      "Test: Total time: 0:07:36 (0.6759 s / it)\n",
      "Averaged stats: model_time: 0.1657 (0.1647)  evaluator_time: 0.1732 (0.4901)\n",
      "Accumulating evaluation results...\n",
      "DONE (t=1.99s).\n",
      "IoU metric: bbox\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.379\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.851\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.255\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.012\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.365\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.442\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.014\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.130\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.459\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.067\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.451\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.509\n",
      "Epoch: [1]  [   0/2699]  eta: 1:21:07  lr: 0.005000  loss: 0.7466 (0.7466)  loss_classifier: 0.2871 (0.2871)  loss_box_reg: 0.2353 (0.2353)  loss_objectness: 0.1133 (0.1133)  loss_rpn_box_reg: 0.1109 (0.1109)  time: 1.8033  data: 1.1391  max mem: 6482\n",
      "Epoch: [1]  [ 100/2699]  eta: 0:22:52  lr: 0.005000  loss: 0.5543 (0.5879)  loss_classifier: 0.2191 (0.2250)  loss_box_reg: 0.2240 (0.2290)  loss_objectness: 0.0554 (0.0571)  loss_rpn_box_reg: 0.0774 (0.0768)  time: 0.5157  data: 0.0119  max mem: 6482\n",
      "Epoch: [1]  [ 200/2699]  eta: 0:21:42  lr: 0.005000  loss: 0.6335 (0.5825)  loss_classifier: 0.2362 (0.2247)  loss_box_reg: 0.2260 (0.2258)  loss_objectness: 0.0614 (0.0558)  loss_rpn_box_reg: 0.0725 (0.0763)  time: 0.5117  data: 0.0112  max mem: 6482\n",
      "Epoch: [1]  [ 300/2699]  eta: 0:20:44  lr: 0.005000  loss: 0.6139 (0.5730)  loss_classifier: 0.2387 (0.2236)  loss_box_reg: 0.2280 (0.2229)  loss_objectness: 0.0512 (0.0535)  loss_rpn_box_reg: 0.0662 (0.0730)  time: 0.5123  data: 0.0108  max mem: 6482\n",
      "Epoch: [1]  [ 400/2699]  eta: 0:19:49  lr: 0.005000  loss: 0.5513 (0.5685)  loss_classifier: 0.2159 (0.2216)  loss_box_reg: 0.2187 (0.2213)  loss_objectness: 0.0441 (0.0529)  loss_rpn_box_reg: 0.0695 (0.0727)  time: 0.5133  data: 0.0117  max mem: 6482\n",
      "Epoch: [1]  [ 500/2699]  eta: 0:18:56  lr: 0.005000  loss: 0.5521 (0.5655)  loss_classifier: 0.2150 (0.2206)  loss_box_reg: 0.2141 (0.2201)  loss_objectness: 0.0441 (0.0526)  loss_rpn_box_reg: 0.0678 (0.0723)  time: 0.5225  data: 0.0120  max mem: 6482\n",
      "Epoch: [1]  [ 600/2699]  eta: 0:18:02  lr: 0.005000  loss: 0.5605 (0.5636)  loss_classifier: 0.2125 (0.2201)  loss_box_reg: 0.2193 (0.2189)  loss_objectness: 0.0550 (0.0524)  loss_rpn_box_reg: 0.0765 (0.0722)  time: 0.5145  data: 0.0114  max mem: 6482\n",
      "Epoch: [1]  [ 700/2699]  eta: 0:17:10  lr: 0.005000  loss: 0.5835 (0.5632)  loss_classifier: 0.2362 (0.2195)  loss_box_reg: 0.2330 (0.2185)  loss_objectness: 0.0496 (0.0526)  loss_rpn_box_reg: 0.0660 (0.0726)  time: 0.5100  data: 0.0107  max mem: 6482\n",
      "Epoch: [1]  [ 800/2699]  eta: 0:16:18  lr: 0.005000  loss: 0.5218 (0.5614)  loss_classifier: 0.2000 (0.2192)  loss_box_reg: 0.2072 (0.2177)  loss_objectness: 0.0403 (0.0524)  loss_rpn_box_reg: 0.0648 (0.0720)  time: 0.5097  data: 0.0108  max mem: 6482\n",
      "Epoch: [1]  [ 900/2699]  eta: 0:15:27  lr: 0.005000  loss: 0.5779 (0.5617)  loss_classifier: 0.2284 (0.2196)  loss_box_reg: 0.2134 (0.2172)  loss_objectness: 0.0500 (0.0527)  loss_rpn_box_reg: 0.0719 (0.0722)  time: 0.5093  data: 0.0108  max mem: 6482\n",
      "Epoch: [1]  [1000/2699]  eta: 0:14:35  lr: 0.005000  loss: 0.5440 (0.5622)  loss_classifier: 0.1946 (0.2199)  loss_box_reg: 0.2212 (0.2172)  loss_objectness: 0.0449 (0.0528)  loss_rpn_box_reg: 0.0617 (0.0723)  time: 0.5085  data: 0.0112  max mem: 6482\n",
      "Epoch: [1]  [1100/2699]  eta: 0:13:44  lr: 0.005000  loss: 0.5137 (0.5611)  loss_classifier: 0.2013 (0.2195)  loss_box_reg: 0.2035 (0.2169)  loss_objectness: 0.0359 (0.0522)  loss_rpn_box_reg: 0.0676 (0.0725)  time: 0.5119  data: 0.0116  max mem: 6482\n",
      "Epoch: [1]  [1200/2699]  eta: 0:12:52  lr: 0.005000  loss: 0.5077 (0.5593)  loss_classifier: 0.1948 (0.2190)  loss_box_reg: 0.2025 (0.2160)  loss_objectness: 0.0483 (0.0520)  loss_rpn_box_reg: 0.0620 (0.0723)  time: 0.5273  data: 0.0142  max mem: 6482\n",
      "Epoch: [1]  [1300/2699]  eta: 0:12:00  lr: 0.005000  loss: 0.4782 (0.5571)  loss_classifier: 0.1960 (0.2183)  loss_box_reg: 0.1968 (0.2153)  loss_objectness: 0.0306 (0.0516)  loss_rpn_box_reg: 0.0567 (0.0719)  time: 0.5092  data: 0.0114  max mem: 6482\n",
      "Epoch: [1]  [1400/2699]  eta: 0:11:08  lr: 0.005000  loss: 0.5079 (0.5560)  loss_classifier: 0.1915 (0.2179)  loss_box_reg: 0.2057 (0.2150)  loss_objectness: 0.0398 (0.0513)  loss_rpn_box_reg: 0.0576 (0.0718)  time: 0.5117  data: 0.0116  max mem: 6482\n",
      "Epoch: [1]  [1500/2699]  eta: 0:10:17  lr: 0.005000  loss: 0.5353 (0.5552)  loss_classifier: 0.1979 (0.2178)  loss_box_reg: 0.1973 (0.2146)  loss_objectness: 0.0419 (0.0511)  loss_rpn_box_reg: 0.0607 (0.0717)  time: 0.5108  data: 0.0109  max mem: 6482\n",
      "Epoch: [1]  [1600/2699]  eta: 0:09:25  lr: 0.005000  loss: 0.4999 (0.5550)  loss_classifier: 0.2079 (0.2179)  loss_box_reg: 0.1993 (0.2143)  loss_objectness: 0.0428 (0.0511)  loss_rpn_box_reg: 0.0654 (0.0717)  time: 0.5113  data: 0.0107  max mem: 6482\n",
      "Epoch: [1]  [1700/2699]  eta: 0:08:34  lr: 0.005000  loss: 0.5789 (0.5545)  loss_classifier: 0.2123 (0.2175)  loss_box_reg: 0.2273 (0.2142)  loss_objectness: 0.0534 (0.0510)  loss_rpn_box_reg: 0.0667 (0.0718)  time: 0.5130  data: 0.0122  max mem: 6482\n",
      "Epoch: [1]  [1800/2699]  eta: 0:07:42  lr: 0.005000  loss: 0.5884 (0.5536)  loss_classifier: 0.2201 (0.2173)  loss_box_reg: 0.2121 (0.2138)  loss_objectness: 0.0460 (0.0508)  loss_rpn_box_reg: 0.0813 (0.0717)  time: 0.5139  data: 0.0113  max mem: 6482\n",
      "Epoch: [1]  [1900/2699]  eta: 0:06:51  lr: 0.005000  loss: 0.5832 (0.5536)  loss_classifier: 0.2260 (0.2173)  loss_box_reg: 0.2234 (0.2137)  loss_objectness: 0.0494 (0.0509)  loss_rpn_box_reg: 0.0684 (0.0717)  time: 0.5273  data: 0.0130  max mem: 6482\n",
      "Epoch: [1]  [2000/2699]  eta: 0:05:59  lr: 0.005000  loss: 0.5146 (0.5533)  loss_classifier: 0.2128 (0.2173)  loss_box_reg: 0.2031 (0.2138)  loss_objectness: 0.0338 (0.0506)  loss_rpn_box_reg: 0.0471 (0.0716)  time: 0.5122  data: 0.0108  max mem: 6482\n",
      "Epoch: [1]  [2100/2699]  eta: 0:05:08  lr: 0.005000  loss: 0.5099 (0.5523)  loss_classifier: 0.1941 (0.2169)  loss_box_reg: 0.1903 (0.2134)  loss_objectness: 0.0437 (0.0505)  loss_rpn_box_reg: 0.0641 (0.0715)  time: 0.5110  data: 0.0108  max mem: 6482\n",
      "Epoch: [1]  [2200/2699]  eta: 0:04:16  lr: 0.005000  loss: 0.4930 (0.5520)  loss_classifier: 0.1830 (0.2168)  loss_box_reg: 0.1906 (0.2132)  loss_objectness: 0.0423 (0.0505)  loss_rpn_box_reg: 0.0565 (0.0715)  time: 0.5123  data: 0.0112  max mem: 6482\n",
      "Epoch: [1]  [2300/2699]  eta: 0:03:25  lr: 0.005000  loss: 0.5268 (0.5512)  loss_classifier: 0.2149 (0.2166)  loss_box_reg: 0.2207 (0.2130)  loss_objectness: 0.0400 (0.0503)  loss_rpn_box_reg: 0.0598 (0.0713)  time: 0.5086  data: 0.0106  max mem: 6482\n",
      "Epoch: [1]  [2400/2699]  eta: 0:02:33  lr: 0.005000  loss: 0.4797 (0.5499)  loss_classifier: 0.1922 (0.2160)  loss_box_reg: 0.1934 (0.2125)  loss_objectness: 0.0434 (0.0502)  loss_rpn_box_reg: 0.0663 (0.0712)  time: 0.5105  data: 0.0111  max mem: 6482\n",
      "Epoch: [1]  [2500/2699]  eta: 0:01:42  lr: 0.005000  loss: 0.5383 (0.5484)  loss_classifier: 0.2140 (0.2155)  loss_box_reg: 0.2027 (0.2119)  loss_objectness: 0.0374 (0.0500)  loss_rpn_box_reg: 0.0637 (0.0710)  time: 0.5143  data: 0.0119  max mem: 6482\n",
      "Epoch: [1]  [2600/2699]  eta: 0:00:50  lr: 0.005000  loss: 0.4888 (0.5474)  loss_classifier: 0.2020 (0.2152)  loss_box_reg: 0.1897 (0.2116)  loss_objectness: 0.0394 (0.0498)  loss_rpn_box_reg: 0.0640 (0.0708)  time: 0.5213  data: 0.0120  max mem: 6482\n",
      "Epoch: [1]  [2698/2699]  eta: 0:00:00  lr: 0.005000  loss: 0.5009 (0.5467)  loss_classifier: 0.2027 (0.2150)  loss_box_reg: 0.1933 (0.2114)  loss_objectness: 0.0282 (0.0496)  loss_rpn_box_reg: 0.0520 (0.0707)  time: 0.4956  data: 0.0105  max mem: 6482\n",
      "Epoch: [1] Total time: 0:23:08 (0.5145 s / it)\n",
      "creating index...\n",
      "index created!\n",
      "Test:  [  0/675]  eta: 0:15:03  model_time: 0.2247 (0.2247)  evaluator_time: 0.3224 (0.3224)  time: 1.3391  data: 0.7690  max mem: 6482\n",
      "Test:  [100/675]  eta: 0:06:22  model_time: 0.1530 (0.1551)  evaluator_time: 0.3749 (0.4834)  time: 0.5539  data: 0.0113  max mem: 6482\n",
      "Test:  [200/675]  eta: 0:05:41  model_time: 0.1554 (0.1550)  evaluator_time: 0.8536 (0.5399)  time: 1.0249  data: 0.0117  max mem: 6482\n",
      "Test:  [300/675]  eta: 0:04:17  model_time: 0.1526 (0.1556)  evaluator_time: 0.5074 (0.5087)  time: 0.6776  data: 0.0110  max mem: 6482\n",
      "Test:  [400/675]  eta: 0:03:03  model_time: 0.1531 (0.1557)  evaluator_time: 0.5039 (0.4903)  time: 0.7614  data: 0.0115  max mem: 6482\n",
      "Test:  [500/675]  eta: 0:01:56  model_time: 0.1561 (0.1557)  evaluator_time: 0.9093 (0.4872)  time: 1.1526  data: 0.0116  max mem: 6482\n",
      "Test:  [600/675]  eta: 0:00:50  model_time: 0.1588 (0.1559)  evaluator_time: 0.2431 (0.4990)  time: 0.4707  data: 0.0108  max mem: 6482\n",
      "Test:  [674/675]  eta: 0:00:00  model_time: 0.1571 (0.1559)  evaluator_time: 0.1818 (0.4819)  time: 0.3696  data: 0.0123  max mem: 6482\n",
      "Test: Total time: 0:07:25 (0.6598 s / it)\n",
      "Averaged stats: model_time: 0.1571 (0.1559)  evaluator_time: 0.1818 (0.4819)\n",
      "Accumulating evaluation results...\n",
      "DONE (t=1.94s).\n",
      "IoU metric: bbox\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.420\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.879\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.330\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.019\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.410\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.467\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.015\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.135\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.493\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.076\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.490\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.524\n",
      "Epoch: [2]  [   0/2699]  eta: 1:35:08  lr: 0.005000  loss: 0.5123 (0.5123)  loss_classifier: 0.1959 (0.1959)  loss_box_reg: 0.2020 (0.2020)  loss_objectness: 0.0275 (0.0275)  loss_rpn_box_reg: 0.0869 (0.0869)  time: 2.1150  data: 1.1648  max mem: 6482\n",
      "Epoch: [2]  [ 100/2699]  eta: 0:22:56  lr: 0.005000  loss: 0.5184 (0.5019)  loss_classifier: 0.2101 (0.2031)  loss_box_reg: 0.1846 (0.1940)  loss_objectness: 0.0424 (0.0393)  loss_rpn_box_reg: 0.0736 (0.0655)  time: 0.5120  data: 0.0109  max mem: 6482\n",
      "Epoch: [2]  [ 200/2699]  eta: 0:21:44  lr: 0.005000  loss: 0.5017 (0.4928)  loss_classifier: 0.1849 (0.1974)  loss_box_reg: 0.1991 (0.1916)  loss_objectness: 0.0359 (0.0394)  loss_rpn_box_reg: 0.0805 (0.0644)  time: 0.5090  data: 0.0107  max mem: 6482\n",
      "Epoch: [2]  [ 300/2699]  eta: 0:20:46  lr: 0.005000  loss: 0.5246 (0.4883)  loss_classifier: 0.1971 (0.1947)  loss_box_reg: 0.2004 (0.1906)  loss_objectness: 0.0380 (0.0394)  loss_rpn_box_reg: 0.0635 (0.0637)  time: 0.5078  data: 0.0106  max mem: 6482\n",
      "Epoch: [2]  [ 400/2699]  eta: 0:19:50  lr: 0.005000  loss: 0.4855 (0.4905)  loss_classifier: 0.1861 (0.1951)  loss_box_reg: 0.1959 (0.1916)  loss_objectness: 0.0300 (0.0398)  loss_rpn_box_reg: 0.0591 (0.0640)  time: 0.5119  data: 0.0106  max mem: 6482\n",
      "Epoch: [2]  [ 500/2699]  eta: 0:18:56  lr: 0.005000  loss: 0.4649 (0.4922)  loss_classifier: 0.1773 (0.1955)  loss_box_reg: 0.1898 (0.1921)  loss_objectness: 0.0339 (0.0399)  loss_rpn_box_reg: 0.0554 (0.0646)  time: 0.5108  data: 0.0111  max mem: 6482\n",
      "Epoch: [2]  [ 600/2699]  eta: 0:18:03  lr: 0.005000  loss: 0.4929 (0.4922)  loss_classifier: 0.2019 (0.1961)  loss_box_reg: 0.1931 (0.1921)  loss_objectness: 0.0371 (0.0396)  loss_rpn_box_reg: 0.0664 (0.0643)  time: 0.5138  data: 0.0113  max mem: 6482\n",
      "Epoch: [2]  [ 700/2699]  eta: 0:17:10  lr: 0.005000  loss: 0.4760 (0.4925)  loss_classifier: 0.1856 (0.1962)  loss_box_reg: 0.1757 (0.1919)  loss_objectness: 0.0316 (0.0399)  loss_rpn_box_reg: 0.0638 (0.0645)  time: 0.5254  data: 0.0146  max mem: 6482\n",
      "Epoch: [2]  [ 800/2699]  eta: 0:16:17  lr: 0.005000  loss: 0.5452 (0.4968)  loss_classifier: 0.2279 (0.1980)  loss_box_reg: 0.1989 (0.1927)  loss_objectness: 0.0446 (0.0407)  loss_rpn_box_reg: 0.0754 (0.0654)  time: 0.5080  data: 0.0106  max mem: 6482\n",
      "Epoch: [2]  [ 900/2699]  eta: 0:15:25  lr: 0.005000  loss: 0.4774 (0.4950)  loss_classifier: 0.1854 (0.1975)  loss_box_reg: 0.1773 (0.1919)  loss_objectness: 0.0397 (0.0406)  loss_rpn_box_reg: 0.0559 (0.0650)  time: 0.5074  data: 0.0102  max mem: 6482\n",
      "Epoch: [2]  [1000/2699]  eta: 0:14:34  lr: 0.005000  loss: 0.4750 (0.4933)  loss_classifier: 0.1893 (0.1968)  loss_box_reg: 0.1984 (0.1914)  loss_objectness: 0.0397 (0.0404)  loss_rpn_box_reg: 0.0604 (0.0647)  time: 0.5087  data: 0.0102  max mem: 6482\n",
      "Epoch: [2]  [1100/2699]  eta: 0:13:42  lr: 0.005000  loss: 0.4927 (0.4948)  loss_classifier: 0.1988 (0.1976)  loss_box_reg: 0.1934 (0.1917)  loss_objectness: 0.0358 (0.0407)  loss_rpn_box_reg: 0.0590 (0.0648)  time: 0.5074  data: 0.0103  max mem: 6482\n",
      "Epoch: [2]  [1200/2699]  eta: 0:12:50  lr: 0.005000  loss: 0.4848 (0.4939)  loss_classifier: 0.1775 (0.1969)  loss_box_reg: 0.2000 (0.1912)  loss_objectness: 0.0421 (0.0408)  loss_rpn_box_reg: 0.0621 (0.0649)  time: 0.5076  data: 0.0103  max mem: 6482\n",
      "Epoch: [2]  [1300/2699]  eta: 0:11:59  lr: 0.005000  loss: 0.4422 (0.4926)  loss_classifier: 0.1597 (0.1964)  loss_box_reg: 0.1714 (0.1907)  loss_objectness: 0.0389 (0.0408)  loss_rpn_box_reg: 0.0593 (0.0647)  time: 0.5148  data: 0.0111  max mem: 6482\n",
      "Epoch: [2]  [1400/2699]  eta: 0:11:07  lr: 0.005000  loss: 0.4843 (0.4920)  loss_classifier: 0.1881 (0.1959)  loss_box_reg: 0.2029 (0.1908)  loss_objectness: 0.0352 (0.0407)  loss_rpn_box_reg: 0.0559 (0.0646)  time: 0.5192  data: 0.0128  max mem: 6482\n",
      "Epoch: [2]  [1500/2699]  eta: 0:10:15  lr: 0.005000  loss: 0.4847 (0.4923)  loss_classifier: 0.1974 (0.1961)  loss_box_reg: 0.1840 (0.1908)  loss_objectness: 0.0341 (0.0407)  loss_rpn_box_reg: 0.0601 (0.0647)  time: 0.5091  data: 0.0102  max mem: 6482\n",
      "Epoch: [2]  [1600/2699]  eta: 0:09:24  lr: 0.005000  loss: 0.4094 (0.4918)  loss_classifier: 0.1725 (0.1959)  loss_box_reg: 0.1613 (0.1906)  loss_objectness: 0.0309 (0.0407)  loss_rpn_box_reg: 0.0527 (0.0646)  time: 0.5089  data: 0.0103  max mem: 6482\n",
      "Epoch: [2]  [1700/2699]  eta: 0:08:32  lr: 0.005000  loss: 0.4556 (0.4905)  loss_classifier: 0.1791 (0.1953)  loss_box_reg: 0.1750 (0.1901)  loss_objectness: 0.0332 (0.0406)  loss_rpn_box_reg: 0.0538 (0.0644)  time: 0.5103  data: 0.0104  max mem: 6482\n",
      "Epoch: [2]  [1800/2699]  eta: 0:07:41  lr: 0.005000  loss: 0.4729 (0.4917)  loss_classifier: 0.1841 (0.1958)  loss_box_reg: 0.1924 (0.1904)  loss_objectness: 0.0349 (0.0409)  loss_rpn_box_reg: 0.0586 (0.0646)  time: 0.5092  data: 0.0106  max mem: 6482\n",
      "Epoch: [2]  [1900/2699]  eta: 0:06:50  lr: 0.005000  loss: 0.5009 (0.4913)  loss_classifier: 0.1901 (0.1957)  loss_box_reg: 0.1933 (0.1900)  loss_objectness: 0.0442 (0.0409)  loss_rpn_box_reg: 0.0685 (0.0646)  time: 0.5137  data: 0.0113  max mem: 6482\n",
      "Epoch: [2]  [2000/2699]  eta: 0:05:58  lr: 0.005000  loss: 0.4991 (0.4913)  loss_classifier: 0.1936 (0.1956)  loss_box_reg: 0.1951 (0.1901)  loss_objectness: 0.0421 (0.0409)  loss_rpn_box_reg: 0.0706 (0.0647)  time: 0.5261  data: 0.0116  max mem: 6482\n",
      "Epoch: [2]  [2100/2699]  eta: 0:05:07  lr: 0.005000  loss: 0.4921 (0.4908)  loss_classifier: 0.1873 (0.1953)  loss_box_reg: 0.1887 (0.1898)  loss_objectness: 0.0323 (0.0410)  loss_rpn_box_reg: 0.0630 (0.0647)  time: 0.5165  data: 0.0114  max mem: 6482\n",
      "Epoch: [2]  [2200/2699]  eta: 0:04:15  lr: 0.005000  loss: 0.5037 (0.4907)  loss_classifier: 0.1946 (0.1952)  loss_box_reg: 0.1992 (0.1898)  loss_objectness: 0.0385 (0.0409)  loss_rpn_box_reg: 0.0572 (0.0647)  time: 0.5074  data: 0.0102  max mem: 6482\n",
      "Epoch: [2]  [2300/2699]  eta: 0:03:24  lr: 0.005000  loss: 0.4320 (0.4901)  loss_classifier: 0.1652 (0.1951)  loss_box_reg: 0.1805 (0.1897)  loss_objectness: 0.0313 (0.0408)  loss_rpn_box_reg: 0.0462 (0.0645)  time: 0.5090  data: 0.0107  max mem: 6482\n",
      "Epoch: [2]  [2400/2699]  eta: 0:02:33  lr: 0.005000  loss: 0.4894 (0.4899)  loss_classifier: 0.2022 (0.1950)  loss_box_reg: 0.1895 (0.1896)  loss_objectness: 0.0455 (0.0408)  loss_rpn_box_reg: 0.0708 (0.0645)  time: 0.5074  data: 0.0104  max mem: 6482\n",
      "Epoch: [2]  [2500/2699]  eta: 0:01:42  lr: 0.005000  loss: 0.5353 (0.4905)  loss_classifier: 0.2094 (0.1953)  loss_box_reg: 0.2023 (0.1896)  loss_objectness: 0.0417 (0.0409)  loss_rpn_box_reg: 0.0794 (0.0646)  time: 0.5121  data: 0.0115  max mem: 6482\n",
      "Epoch: [2]  [2600/2699]  eta: 0:00:50  lr: 0.005000  loss: 0.4575 (0.4907)  loss_classifier: 0.1789 (0.1954)  loss_box_reg: 0.1963 (0.1897)  loss_objectness: 0.0306 (0.0409)  loss_rpn_box_reg: 0.0557 (0.0646)  time: 0.5094  data: 0.0113  max mem: 6482\n",
      "Epoch: [2]  [2698/2699]  eta: 0:00:00  lr: 0.005000  loss: 0.4782 (0.4900)  loss_classifier: 0.1984 (0.1951)  loss_box_reg: 0.1895 (0.1894)  loss_objectness: 0.0312 (0.0409)  loss_rpn_box_reg: 0.0584 (0.0646)  time: 0.5014  data: 0.0109  max mem: 6482\n",
      "Epoch: [2] Total time: 0:23:03 (0.5126 s / it)\n",
      "creating index...\n",
      "index created!\n",
      "Test:  [  0/675]  eta: 0:14:24  model_time: 0.2231 (0.2231)  evaluator_time: 0.2728 (0.2728)  time: 1.2814  data: 0.7666  max mem: 6482\n",
      "Test:  [100/675]  eta: 0:05:44  model_time: 0.1521 (0.1545)  evaluator_time: 0.3076 (0.4168)  time: 0.5003  data: 0.0112  max mem: 6482\n",
      "Test:  [200/675]  eta: 0:05:10  model_time: 0.1549 (0.1544)  evaluator_time: 0.7352 (0.4767)  time: 0.9163  data: 0.0106  max mem: 6482\n",
      "Test:  [300/675]  eta: 0:03:53  model_time: 0.1516 (0.1549)  evaluator_time: 0.4157 (0.4453)  time: 0.5971  data: 0.0117  max mem: 6482\n",
      "Test:  [400/675]  eta: 0:02:45  model_time: 0.1525 (0.1551)  evaluator_time: 0.4299 (0.4264)  time: 0.6787  data: 0.0105  max mem: 6482\n",
      "Test:  [500/675]  eta: 0:01:45  model_time: 0.1549 (0.1551)  evaluator_time: 0.8019 (0.4248)  time: 1.0187  data: 0.0108  max mem: 6482\n",
      "Test:  [600/675]  eta: 0:00:45  model_time: 0.1606 (0.1553)  evaluator_time: 0.2115 (0.4353)  time: 0.4294  data: 0.0112  max mem: 6482\n",
      "Test:  [674/675]  eta: 0:00:00  model_time: 0.1583 (0.1553)  evaluator_time: 0.1295 (0.4195)  time: 0.3063  data: 0.0103  max mem: 6482\n",
      "Test: Total time: 0:06:42 (0.5958 s / it)\n",
      "Averaged stats: model_time: 0.1583 (0.1553)  evaluator_time: 0.1295 (0.4195)\n",
      "Accumulating evaluation results...\n",
      "DONE (t=1.68s).\n",
      "IoU metric: bbox\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.437\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.886\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.370\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.044\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.425\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.501\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.015\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.141\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.507\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.122\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.496\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.570\n",
      "Epoch: [3]  [   0/2699]  eta: 1:05:20  lr: 0.000500  loss: 0.2028 (0.2028)  loss_classifier: 0.0791 (0.0791)  loss_box_reg: 0.0795 (0.0795)  loss_objectness: 0.0172 (0.0172)  loss_rpn_box_reg: 0.0270 (0.0270)  time: 1.4525  data: 0.7730  max mem: 6482\n",
      "Epoch: [3]  [ 100/2699]  eta: 0:22:37  lr: 0.000500  loss: 0.4098 (0.4227)  loss_classifier: 0.1621 (0.1734)  loss_box_reg: 0.1608 (0.1658)  loss_objectness: 0.0253 (0.0314)  loss_rpn_box_reg: 0.0488 (0.0522)  time: 0.5079  data: 0.0100  max mem: 6482\n",
      "Epoch: [3]  [ 200/2699]  eta: 0:21:33  lr: 0.000500  loss: 0.4271 (0.4330)  loss_classifier: 0.1727 (0.1762)  loss_box_reg: 0.1606 (0.1669)  loss_objectness: 0.0299 (0.0345)  loss_rpn_box_reg: 0.0596 (0.0554)  time: 0.5268  data: 0.0111  max mem: 6482\n",
      "Epoch: [3]  [ 300/2699]  eta: 0:20:34  lr: 0.000500  loss: 0.3818 (0.4319)  loss_classifier: 0.1723 (0.1759)  loss_box_reg: 0.1637 (0.1658)  loss_objectness: 0.0382 (0.0344)  loss_rpn_box_reg: 0.0482 (0.0558)  time: 0.5093  data: 0.0113  max mem: 6482\n",
      "Epoch: [3]  [ 400/2699]  eta: 0:19:40  lr: 0.000500  loss: 0.4167 (0.4285)  loss_classifier: 0.1656 (0.1745)  loss_box_reg: 0.1594 (0.1645)  loss_objectness: 0.0280 (0.0340)  loss_rpn_box_reg: 0.0486 (0.0554)  time: 0.5061  data: 0.0101  max mem: 6482\n",
      "Epoch: [3]  [ 500/2699]  eta: 0:18:47  lr: 0.000500  loss: 0.3962 (0.4227)  loss_classifier: 0.1660 (0.1725)  loss_box_reg: 0.1566 (0.1624)  loss_objectness: 0.0256 (0.0334)  loss_rpn_box_reg: 0.0427 (0.0544)  time: 0.5065  data: 0.0106  max mem: 6482\n",
      "Epoch: [3]  [ 600/2699]  eta: 0:17:55  lr: 0.000500  loss: 0.4505 (0.4228)  loss_classifier: 0.1770 (0.1718)  loss_box_reg: 0.1839 (0.1630)  loss_objectness: 0.0299 (0.0334)  loss_rpn_box_reg: 0.0560 (0.0547)  time: 0.5089  data: 0.0110  max mem: 6482\n",
      "Epoch: [3]  [ 700/2699]  eta: 0:17:04  lr: 0.000500  loss: 0.4039 (0.4212)  loss_classifier: 0.1646 (0.1711)  loss_box_reg: 0.1624 (0.1628)  loss_objectness: 0.0247 (0.0330)  loss_rpn_box_reg: 0.0488 (0.0543)  time: 0.5080  data: 0.0099  max mem: 6482\n",
      "Epoch: [3]  [ 800/2699]  eta: 0:16:13  lr: 0.000500  loss: 0.4088 (0.4215)  loss_classifier: 0.1563 (0.1711)  loss_box_reg: 0.1655 (0.1629)  loss_objectness: 0.0293 (0.0330)  loss_rpn_box_reg: 0.0549 (0.0545)  time: 0.5200  data: 0.0117  max mem: 6482\n",
      "Epoch: [3]  [ 900/2699]  eta: 0:15:21  lr: 0.000500  loss: 0.4508 (0.4202)  loss_classifier: 0.1685 (0.1707)  loss_box_reg: 0.1580 (0.1623)  loss_objectness: 0.0252 (0.0327)  loss_rpn_box_reg: 0.0633 (0.0545)  time: 0.5225  data: 0.0121  max mem: 6482\n",
      "Epoch: [3]  [1000/2699]  eta: 0:14:29  lr: 0.000500  loss: 0.3859 (0.4192)  loss_classifier: 0.1624 (0.1705)  loss_box_reg: 0.1582 (0.1619)  loss_objectness: 0.0309 (0.0324)  loss_rpn_box_reg: 0.0445 (0.0544)  time: 0.5065  data: 0.0101  max mem: 6482\n",
      "Epoch: [3]  [1100/2699]  eta: 0:13:38  lr: 0.000500  loss: 0.4086 (0.4184)  loss_classifier: 0.1583 (0.1702)  loss_box_reg: 0.1611 (0.1617)  loss_objectness: 0.0287 (0.0322)  loss_rpn_box_reg: 0.0417 (0.0544)  time: 0.5107  data: 0.0107  max mem: 6482\n",
      "Epoch: [3]  [1200/2699]  eta: 0:12:47  lr: 0.000500  loss: 0.4035 (0.4181)  loss_classifier: 0.1680 (0.1699)  loss_box_reg: 0.1571 (0.1619)  loss_objectness: 0.0268 (0.0320)  loss_rpn_box_reg: 0.0510 (0.0543)  time: 0.5091  data: 0.0102  max mem: 6482\n",
      "Epoch: [3]  [1300/2699]  eta: 0:11:56  lr: 0.000500  loss: 0.5048 (0.4185)  loss_classifier: 0.1977 (0.1700)  loss_box_reg: 0.1891 (0.1621)  loss_objectness: 0.0328 (0.0320)  loss_rpn_box_reg: 0.0693 (0.0544)  time: 0.5059  data: 0.0103  max mem: 6482\n",
      "Epoch: [3]  [1400/2699]  eta: 0:11:04  lr: 0.000500  loss: 0.4341 (0.4182)  loss_classifier: 0.1719 (0.1698)  loss_box_reg: 0.1630 (0.1620)  loss_objectness: 0.0295 (0.0320)  loss_rpn_box_reg: 0.0481 (0.0543)  time: 0.5090  data: 0.0105  max mem: 6482\n",
      "Epoch: [3]  [1500/2699]  eta: 0:10:13  lr: 0.000500  loss: 0.3909 (0.4192)  loss_classifier: 0.1677 (0.1702)  loss_box_reg: 0.1648 (0.1623)  loss_objectness: 0.0342 (0.0323)  loss_rpn_box_reg: 0.0561 (0.0546)  time: 0.5102  data: 0.0115  max mem: 6482\n",
      "Epoch: [3]  [1600/2699]  eta: 0:09:22  lr: 0.000500  loss: 0.4179 (0.4193)  loss_classifier: 0.1726 (0.1700)  loss_box_reg: 0.1635 (0.1622)  loss_objectness: 0.0256 (0.0322)  loss_rpn_box_reg: 0.0571 (0.0548)  time: 0.5227  data: 0.0108  max mem: 6482\n",
      "Epoch: [3]  [1700/2699]  eta: 0:08:31  lr: 0.000500  loss: 0.3708 (0.4190)  loss_classifier: 0.1567 (0.1699)  loss_box_reg: 0.1586 (0.1621)  loss_objectness: 0.0218 (0.0322)  loss_rpn_box_reg: 0.0493 (0.0548)  time: 0.5101  data: 0.0105  max mem: 6482\n",
      "Epoch: [3]  [1800/2699]  eta: 0:07:39  lr: 0.000500  loss: 0.4196 (0.4179)  loss_classifier: 0.1659 (0.1695)  loss_box_reg: 0.1592 (0.1617)  loss_objectness: 0.0246 (0.0320)  loss_rpn_box_reg: 0.0548 (0.0546)  time: 0.5092  data: 0.0104  max mem: 6482\n",
      "Epoch: [3]  [1900/2699]  eta: 0:06:48  lr: 0.000500  loss: 0.4320 (0.4167)  loss_classifier: 0.1749 (0.1691)  loss_box_reg: 0.1547 (0.1613)  loss_objectness: 0.0258 (0.0319)  loss_rpn_box_reg: 0.0458 (0.0544)  time: 0.5083  data: 0.0101  max mem: 6482\n",
      "Epoch: [3]  [2000/2699]  eta: 0:05:57  lr: 0.000500  loss: 0.4132 (0.4162)  loss_classifier: 0.1673 (0.1689)  loss_box_reg: 0.1510 (0.1610)  loss_objectness: 0.0228 (0.0318)  loss_rpn_box_reg: 0.0554 (0.0544)  time: 0.5069  data: 0.0100  max mem: 6482\n",
      "Epoch: [3]  [2100/2699]  eta: 0:05:06  lr: 0.000500  loss: 0.4126 (0.4155)  loss_classifier: 0.1576 (0.1687)  loss_box_reg: 0.1503 (0.1607)  loss_objectness: 0.0236 (0.0318)  loss_rpn_box_reg: 0.0522 (0.0544)  time: 0.5088  data: 0.0108  max mem: 6482\n",
      "Epoch: [3]  [2200/2699]  eta: 0:04:15  lr: 0.000500  loss: 0.3799 (0.4139)  loss_classifier: 0.1505 (0.1681)  loss_box_reg: 0.1586 (0.1601)  loss_objectness: 0.0251 (0.0315)  loss_rpn_box_reg: 0.0520 (0.0542)  time: 0.5187  data: 0.0108  max mem: 6482\n",
      "Epoch: [3]  [2300/2699]  eta: 0:03:24  lr: 0.000500  loss: 0.3500 (0.4130)  loss_classifier: 0.1552 (0.1678)  loss_box_reg: 0.1416 (0.1598)  loss_objectness: 0.0223 (0.0314)  loss_rpn_box_reg: 0.0502 (0.0541)  time: 0.5131  data: 0.0105  max mem: 6482\n",
      "Epoch: [3]  [2400/2699]  eta: 0:02:32  lr: 0.000500  loss: 0.3689 (0.4129)  loss_classifier: 0.1521 (0.1677)  loss_box_reg: 0.1454 (0.1597)  loss_objectness: 0.0226 (0.0313)  loss_rpn_box_reg: 0.0474 (0.0541)  time: 0.5085  data: 0.0106  max mem: 6482\n",
      "Epoch: [3]  [2500/2699]  eta: 0:01:41  lr: 0.000500  loss: 0.3979 (0.4129)  loss_classifier: 0.1562 (0.1677)  loss_box_reg: 0.1526 (0.1598)  loss_objectness: 0.0207 (0.0313)  loss_rpn_box_reg: 0.0516 (0.0541)  time: 0.5099  data: 0.0106  max mem: 6482\n",
      "Epoch: [3]  [2600/2699]  eta: 0:00:50  lr: 0.000500  loss: 0.4601 (0.4122)  loss_classifier: 0.1781 (0.1674)  loss_box_reg: 0.1650 (0.1597)  loss_objectness: 0.0300 (0.0311)  loss_rpn_box_reg: 0.0580 (0.0540)  time: 0.5080  data: 0.0107  max mem: 6482\n",
      "Epoch: [3]  [2698/2699]  eta: 0:00:00  lr: 0.000500  loss: 0.3828 (0.4118)  loss_classifier: 0.1402 (0.1672)  loss_box_reg: 0.1516 (0.1595)  loss_objectness: 0.0221 (0.0311)  loss_rpn_box_reg: 0.0537 (0.0540)  time: 0.4933  data: 0.0101  max mem: 6482\n",
      "Epoch: [3] Total time: 0:23:00 (0.5116 s / it)\n",
      "creating index...\n",
      "index created!\n",
      "Test:  [  0/675]  eta: 0:16:08  model_time: 0.2790 (0.2790)  evaluator_time: 0.4726 (0.4726)  time: 1.4349  data: 0.6569  max mem: 6482\n",
      "Test:  [100/675]  eta: 0:05:20  model_time: 0.1468 (0.1498)  evaluator_time: 0.2787 (0.3818)  time: 0.4560  data: 0.0107  max mem: 6482\n",
      "Test:  [200/675]  eta: 0:04:56  model_time: 0.1462 (0.1493)  evaluator_time: 0.6988 (0.4517)  time: 0.9122  data: 0.0107  max mem: 6482\n",
      "Test:  [300/675]  eta: 0:03:43  model_time: 0.1456 (0.1500)  evaluator_time: 0.3875 (0.4234)  time: 0.5703  data: 0.0109  max mem: 6482\n",
      "Test:  [400/675]  eta: 0:02:37  model_time: 0.1463 (0.1501)  evaluator_time: 0.3917 (0.4021)  time: 0.6318  data: 0.0112  max mem: 6482\n",
      "Test:  [500/675]  eta: 0:01:39  model_time: 0.1472 (0.1502)  evaluator_time: 0.8403 (0.3993)  time: 1.0055  data: 0.0105  max mem: 6482\n",
      "Test:  [600/675]  eta: 0:00:43  model_time: 0.1550 (0.1504)  evaluator_time: 0.1850 (0.4121)  time: 0.4095  data: 0.0120  max mem: 6482\n",
      "Test:  [674/675]  eta: 0:00:00  model_time: 0.1538 (0.1505)  evaluator_time: 0.0949 (0.3970)  time: 0.2662  data: 0.0102  max mem: 6482\n",
      "Test: Total time: 0:06:23 (0.5687 s / it)\n",
      "Averaged stats: model_time: 0.1538 (0.1505)  evaluator_time: 0.0949 (0.3970)\n",
      "Accumulating evaluation results...\n",
      "DONE (t=1.57s).\n",
      "IoU metric: bbox\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.466\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.900\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.424\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.053\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.456\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.531\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.016\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.147\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.535\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.130\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.525\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.595\n",
      "Epoch: [4]  [   0/2699]  eta: 1:04:59  lr: 0.000500  loss: 0.3214 (0.3214)  loss_classifier: 0.1388 (0.1388)  loss_box_reg: 0.1300 (0.1300)  loss_objectness: 0.0222 (0.0222)  loss_rpn_box_reg: 0.0304 (0.0304)  time: 1.4448  data: 0.7566  max mem: 6482\n",
      "Epoch: [4]  [ 100/2699]  eta: 0:22:36  lr: 0.000500  loss: 0.4236 (0.4013)  loss_classifier: 0.1607 (0.1630)  loss_box_reg: 0.1629 (0.1579)  loss_objectness: 0.0304 (0.0279)  loss_rpn_box_reg: 0.0539 (0.0526)  time: 0.5073  data: 0.0100  max mem: 6482\n",
      "Epoch: [4]  [ 200/2699]  eta: 0:21:32  lr: 0.000500  loss: 0.3789 (0.3992)  loss_classifier: 0.1513 (0.1617)  loss_box_reg: 0.1651 (0.1561)  loss_objectness: 0.0262 (0.0286)  loss_rpn_box_reg: 0.0514 (0.0528)  time: 0.5214  data: 0.0118  max mem: 6482\n",
      "Epoch: [4]  [ 300/2699]  eta: 0:20:33  lr: 0.000500  loss: 0.3853 (0.3956)  loss_classifier: 0.1602 (0.1607)  loss_box_reg: 0.1492 (0.1541)  loss_objectness: 0.0240 (0.0283)  loss_rpn_box_reg: 0.0588 (0.0525)  time: 0.5078  data: 0.0104  max mem: 6482\n",
      "Epoch: [4]  [ 400/2699]  eta: 0:19:40  lr: 0.000500  loss: 0.3958 (0.3941)  loss_classifier: 0.1584 (0.1599)  loss_box_reg: 0.1549 (0.1538)  loss_objectness: 0.0340 (0.0283)  loss_rpn_box_reg: 0.0544 (0.0522)  time: 0.5077  data: 0.0101  max mem: 6482\n",
      "Epoch: [4]  [ 500/2699]  eta: 0:18:49  lr: 0.000500  loss: 0.3642 (0.3953)  loss_classifier: 0.1512 (0.1600)  loss_box_reg: 0.1496 (0.1543)  loss_objectness: 0.0209 (0.0286)  loss_rpn_box_reg: 0.0493 (0.0524)  time: 0.5114  data: 0.0113  max mem: 6482\n",
      "Epoch: [4]  [ 600/2699]  eta: 0:17:56  lr: 0.000500  loss: 0.3845 (0.3935)  loss_classifier: 0.1540 (0.1589)  loss_box_reg: 0.1486 (0.1543)  loss_objectness: 0.0249 (0.0284)  loss_rpn_box_reg: 0.0502 (0.0519)  time: 0.5065  data: 0.0103  max mem: 6482\n",
      "Epoch: [4]  [ 700/2699]  eta: 0:17:04  lr: 0.000500  loss: 0.3787 (0.3927)  loss_classifier: 0.1544 (0.1586)  loss_box_reg: 0.1525 (0.1542)  loss_objectness: 0.0151 (0.0282)  loss_rpn_box_reg: 0.0402 (0.0517)  time: 0.5073  data: 0.0104  max mem: 6482\n",
      "Epoch: [4]  [ 800/2699]  eta: 0:16:13  lr: 0.000500  loss: 0.4136 (0.3937)  loss_classifier: 0.1762 (0.1594)  loss_box_reg: 0.1678 (0.1541)  loss_objectness: 0.0302 (0.0283)  loss_rpn_box_reg: 0.0563 (0.0519)  time: 0.5190  data: 0.0131  max mem: 6482\n",
      "Epoch: [4]  [ 900/2699]  eta: 0:15:21  lr: 0.000500  loss: 0.3545 (0.3939)  loss_classifier: 0.1469 (0.1595)  loss_box_reg: 0.1408 (0.1542)  loss_objectness: 0.0169 (0.0283)  loss_rpn_box_reg: 0.0451 (0.0519)  time: 0.5212  data: 0.0124  max mem: 6482\n",
      "Epoch: [4]  [1000/2699]  eta: 0:14:29  lr: 0.000500  loss: 0.4232 (0.3934)  loss_classifier: 0.1787 (0.1593)  loss_box_reg: 0.1512 (0.1538)  loss_objectness: 0.0241 (0.0284)  loss_rpn_box_reg: 0.0553 (0.0520)  time: 0.5090  data: 0.0110  max mem: 6482\n",
      "Epoch: [4]  [1100/2699]  eta: 0:13:38  lr: 0.000500  loss: 0.3763 (0.3932)  loss_classifier: 0.1593 (0.1594)  loss_box_reg: 0.1494 (0.1537)  loss_objectness: 0.0249 (0.0282)  loss_rpn_box_reg: 0.0452 (0.0520)  time: 0.5079  data: 0.0100  max mem: 6482\n",
      "Epoch: [4]  [1200/2699]  eta: 0:12:47  lr: 0.000500  loss: 0.3744 (0.3939)  loss_classifier: 0.1690 (0.1596)  loss_box_reg: 0.1464 (0.1538)  loss_objectness: 0.0267 (0.0283)  loss_rpn_box_reg: 0.0476 (0.0522)  time: 0.5080  data: 0.0104  max mem: 6482\n",
      "Epoch: [4]  [1300/2699]  eta: 0:11:56  lr: 0.000500  loss: 0.4040 (0.3938)  loss_classifier: 0.1674 (0.1597)  loss_box_reg: 0.1515 (0.1537)  loss_objectness: 0.0259 (0.0283)  loss_rpn_box_reg: 0.0581 (0.0521)  time: 0.5077  data: 0.0107  max mem: 6482\n",
      "Epoch: [4]  [1400/2699]  eta: 0:11:04  lr: 0.000500  loss: 0.3951 (0.3931)  loss_classifier: 0.1613 (0.1596)  loss_box_reg: 0.1450 (0.1533)  loss_objectness: 0.0252 (0.0282)  loss_rpn_box_reg: 0.0541 (0.0521)  time: 0.5113  data: 0.0121  max mem: 6482\n",
      "Epoch: [4]  [1500/2699]  eta: 0:10:13  lr: 0.000500  loss: 0.3993 (0.3930)  loss_classifier: 0.1659 (0.1595)  loss_box_reg: 0.1446 (0.1530)  loss_objectness: 0.0253 (0.0283)  loss_rpn_box_reg: 0.0521 (0.0522)  time: 0.5189  data: 0.0116  max mem: 6482\n",
      "Epoch: [4]  [1600/2699]  eta: 0:09:22  lr: 0.000500  loss: 0.3894 (0.3929)  loss_classifier: 0.1564 (0.1594)  loss_box_reg: 0.1516 (0.1529)  loss_objectness: 0.0227 (0.0283)  loss_rpn_box_reg: 0.0485 (0.0523)  time: 0.5194  data: 0.0107  max mem: 6482\n",
      "Epoch: [4]  [1700/2699]  eta: 0:08:35  lr: 0.000500  loss: 0.3868 (0.3936)  loss_classifier: 0.1628 (0.1596)  loss_box_reg: 0.1465 (0.1531)  loss_objectness: 0.0241 (0.0283)  loss_rpn_box_reg: 0.0548 (0.0525)  time: 0.5140  data: 0.0130  max mem: 6482\n",
      "Epoch: [4]  [1800/2699]  eta: 0:07:43  lr: 0.000500  loss: 0.3917 (0.3935)  loss_classifier: 0.1576 (0.1597)  loss_box_reg: 0.1414 (0.1530)  loss_objectness: 0.0249 (0.0283)  loss_rpn_box_reg: 0.0438 (0.0525)  time: 0.5098  data: 0.0106  max mem: 6482\n",
      "Epoch: [4]  [1900/2699]  eta: 0:06:51  lr: 0.000500  loss: 0.3652 (0.3936)  loss_classifier: 0.1519 (0.1599)  loss_box_reg: 0.1428 (0.1529)  loss_objectness: 0.0199 (0.0283)  loss_rpn_box_reg: 0.0464 (0.0525)  time: 0.5079  data: 0.0106  max mem: 6482\n",
      "Epoch: [4]  [2000/2699]  eta: 0:06:00  lr: 0.000500  loss: 0.3797 (0.3940)  loss_classifier: 0.1422 (0.1600)  loss_box_reg: 0.1552 (0.1532)  loss_objectness: 0.0305 (0.0283)  loss_rpn_box_reg: 0.0434 (0.0526)  time: 0.5088  data: 0.0102  max mem: 6482\n",
      "Epoch: [4]  [2100/2699]  eta: 0:05:08  lr: 0.000500  loss: 0.4188 (0.3941)  loss_classifier: 0.1630 (0.1601)  loss_box_reg: 0.1536 (0.1533)  loss_objectness: 0.0261 (0.0282)  loss_rpn_box_reg: 0.0520 (0.0525)  time: 0.5074  data: 0.0103  max mem: 6482\n",
      "Epoch: [4]  [2200/2699]  eta: 0:04:16  lr: 0.000500  loss: 0.3235 (0.3934)  loss_classifier: 0.1358 (0.1598)  loss_box_reg: 0.1368 (0.1531)  loss_objectness: 0.0239 (0.0281)  loss_rpn_box_reg: 0.0430 (0.0524)  time: 0.5073  data: 0.0104  max mem: 6482\n",
      "Epoch: [4]  [2300/2699]  eta: 0:03:25  lr: 0.000500  loss: 0.3593 (0.3932)  loss_classifier: 0.1481 (0.1596)  loss_box_reg: 0.1455 (0.1530)  loss_objectness: 0.0271 (0.0281)  loss_rpn_box_reg: 0.0499 (0.0524)  time: 0.5171  data: 0.0117  max mem: 6482\n",
      "Epoch: [4]  [2400/2699]  eta: 0:02:33  lr: 0.000500  loss: 0.4715 (0.3938)  loss_classifier: 0.2006 (0.1598)  loss_box_reg: 0.1756 (0.1532)  loss_objectness: 0.0353 (0.0282)  loss_rpn_box_reg: 0.0663 (0.0526)  time: 0.5075  data: 0.0104  max mem: 6482\n",
      "Epoch: [4]  [2500/2699]  eta: 0:01:42  lr: 0.000500  loss: 0.3596 (0.3936)  loss_classifier: 0.1509 (0.1597)  loss_box_reg: 0.1399 (0.1530)  loss_objectness: 0.0267 (0.0282)  loss_rpn_box_reg: 0.0519 (0.0526)  time: 0.5078  data: 0.0103  max mem: 6482\n",
      "Epoch: [4]  [2600/2699]  eta: 0:00:50  lr: 0.000500  loss: 0.2974 (0.3933)  loss_classifier: 0.1161 (0.1595)  loss_box_reg: 0.1351 (0.1530)  loss_objectness: 0.0220 (0.0282)  loss_rpn_box_reg: 0.0326 (0.0526)  time: 0.5113  data: 0.0107  max mem: 6482\n",
      "Epoch: [4]  [2698/2699]  eta: 0:00:00  lr: 0.000500  loss: 0.3662 (0.3930)  loss_classifier: 0.1395 (0.1595)  loss_box_reg: 0.1422 (0.1529)  loss_objectness: 0.0228 (0.0281)  loss_rpn_box_reg: 0.0428 (0.0526)  time: 0.4936  data: 0.0105  max mem: 6482\n",
      "Epoch: [4] Total time: 0:23:07 (0.5140 s / it)\n",
      "creating index...\n",
      "index created!\n",
      "Test:  [  0/675]  eta: 0:17:57  model_time: 0.2342 (0.2342)  evaluator_time: 0.7852 (0.7852)  time: 1.5959  data: 0.5570  max mem: 6482\n",
      "Test:  [100/675]  eta: 0:05:18  model_time: 0.1482 (0.1505)  evaluator_time: 0.2684 (0.3776)  time: 0.4587  data: 0.0112  max mem: 6482\n",
      "Test:  [200/675]  eta: 0:04:54  model_time: 0.1486 (0.1506)  evaluator_time: 0.7155 (0.4471)  time: 0.9258  data: 0.0116  max mem: 6482\n",
      "Test:  [300/675]  eta: 0:03:42  model_time: 0.1495 (0.1516)  evaluator_time: 0.4050 (0.4199)  time: 0.5979  data: 0.0114  max mem: 6482\n",
      "Test:  [400/675]  eta: 0:02:36  model_time: 0.1484 (0.1517)  evaluator_time: 0.3784 (0.3976)  time: 0.6214  data: 0.0112  max mem: 6482\n",
      "Test:  [500/675]  eta: 0:01:39  model_time: 0.1495 (0.1518)  evaluator_time: 0.7556 (0.3945)  time: 1.0048  data: 0.0107  max mem: 6482\n",
      "Test:  [600/675]  eta: 0:00:43  model_time: 0.1576 (0.1519)  evaluator_time: 0.1584 (0.4064)  time: 0.3943  data: 0.0108  max mem: 6482\n",
      "Test:  [674/675]  eta: 0:00:00  model_time: 0.1549 (0.1520)  evaluator_time: 0.0859 (0.3914)  time: 0.2574  data: 0.0104  max mem: 6482\n",
      "Test: Total time: 0:06:20 (0.5644 s / it)\n",
      "Averaged stats: model_time: 0.1549 (0.1520)  evaluator_time: 0.0859 (0.3914)\n",
      "Accumulating evaluation results...\n",
      "DONE (t=1.64s).\n",
      "IoU metric: bbox\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.465\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.899\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.424\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.059\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.455\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.531\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.016\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.147\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.535\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.134\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.524\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.597\n"
     ]
    }
   ],
   "source": [
    "num_classes = 2\n",
    "train_dataset = WheatDataset(train_df, folds=[0, 1, 2, 4])\n",
    "train_loader = data_utils.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, collate_fn=utils.collate_fn)\n",
    "\n",
    "val_dataset = WheatDataset(train_df, folds=[3])\n",
    "val_loader = data_utils.DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, collate_fn=utils.collate_fn)\n",
    "\n",
    "\n",
    "# move model to the right device\n",
    "model.to(DEVICE)\n",
    "\n",
    "# construct an optimizer\n",
    "params = [p for p in model.parameters() if p.requires_grad]\n",
    "optimizer = torch.optim.SGD(params, lr=0.005, momentum=0.9, weight_decay=0.0005)\n",
    "\n",
    "# and a learning rate scheduler\n",
    "lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "    # train for one epoch, printing every 100 iterations\n",
    "    train_one_epoch(model, optimizer, train_loader, DEVICE, epoch, print_freq=100)\n",
    "    # update the learning rate\n",
    "    lr_scheduler.step()\n",
    "    # evaluate on the validation dataset\n",
    "    evaluate(model, val_loader, device=DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'fasterrcnn_resnet101_fold3.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bbox(bboxes, col, color='white'):\n",
    "    \n",
    "    for i in range(len(bboxes)):\n",
    "        # Create a Rectangle patch\n",
    "        rect = patches.Rectangle(\n",
    "            (bboxes[i][0], bboxes[i][1]),\n",
    "            bboxes[i][2] - bboxes[i][0], \n",
    "            bboxes[i][3] - bboxes[i][1], \n",
    "            linewidth=2, \n",
    "            edgecolor=color, \n",
    "            facecolor='none')\n",
    "\n",
    "        # Add the patch to the Axes\n",
    "        col.add_patch(rect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'device' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-e23ba2552ea7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mimage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcvtColor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCOLOR_BGR2RGB\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mimage\u001b[0m \u001b[0;34m/=\u001b[0m \u001b[0;36m255.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mpred_bboxes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'boxes'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'device' is not defined"
     ]
    }
   ],
   "source": [
    "for img in os.listdir(TEST_DIR)[:5]:\n",
    "    image = cv2.imread(os.path.join(TEST_DIR, img), cv2.IMREAD_COLOR)\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n",
    "    image /= 255.0\n",
    "    preds = model([torch.from_numpy(np.transpose(image, (2, 0, 1))).to(device)])[0]\n",
    "    \n",
    "    pred_bboxes = preds['boxes'].cpu().detach().numpy()\n",
    "    pred_scores = preds['scores'].cpu().detach().numpy()\n",
    "    \n",
    "    mask = pred_scores >= 0.4\n",
    "    pred_scores = pred_scores[mask]\n",
    "    pred_bboxes = pred_bboxes[mask]\n",
    "    \n",
    "    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 10))\n",
    "    get_bbox(pred_bboxes, ax, color='red')\n",
    "    ax.imshow(image)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "01a4d0f2ddc041aabea72da47ab68518": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "34e2e6df025f489b8a3fbf31bb7fa7c6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_dc43bb83d6f7492987059310a86ca741",
       "max": 178728960.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_5dbcc699d15f476fb9848c6318bc1afe",
       "value": 178728960.0
      }
     },
     "5c881307f63743468c98a2ddddf500e3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "5dbcc699d15f476fb9848c6318bc1afe": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "6f95d213bfd840fabc3d3d0966a42e49": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_34e2e6df025f489b8a3fbf31bb7fa7c6",
        "IPY_MODEL_adadeff2b93a4023828ae0c5abb9b9e5"
       ],
       "layout": "IPY_MODEL_b39068f400f3402ba5950c5a9bb224e1"
      }
     },
     "adadeff2b93a4023828ae0c5abb9b9e5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_01a4d0f2ddc041aabea72da47ab68518",
       "placeholder": "​",
       "style": "IPY_MODEL_5c881307f63743468c98a2ddddf500e3",
       "value": " 170M/170M [00:16&lt;00:00, 10.6MB/s]"
      }
     },
     "b39068f400f3402ba5950c5a9bb224e1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "dc43bb83d6f7492987059310a86ca741": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
